In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")
#Importing the important library
In [4]:
df=pd.read_excel('file:///C:/Users/Ritesh/Desktop/Notes/Datasets%20for%20Python/NBFC Loan Transaction Data.xlsx')
#Loading of the data
In [5]:
df.head()
#First 5 records
Out[5]:
In [6]:
print(df['AUTHORIZATIONDATE'].min())
print(df['AUTHORIZATIONDATE'].max())
print(df['LAST_RECEIPT_DATE'].min())
print(df['LAST_RECEIPT_DATE'].max())
In [7]:
#We can see that there are total 53 columns. And looking atthe various columns we can see that there are null values present
#in certain columns. More over looking at datatype of the column , datatype looks good.
df.info()
In [8]:
df.shape
Out[8]:
In [9]:
pd.options.display.max_columns = None
#To display all the columns.
In [11]:
df.describe().T
Out[11]:
In [12]:
df[df['LOAN_AMT'] - df['NET_DISBURSED_AMT'] > 0].head(10)
Out[12]:
Establishing relationship between different columns . Below are the columns which are derived from the other columns. This information¶
will help us in performing feature engineering.¶
PRE_EMI_DUEAMT= PRE_EMI_RECEIVED_AMT+PRE_EMI_OS_AMOUNT
OUTSTANDING_PRINCIPAL+ PAID_PRINCIPAL = LOAN_AMT
BALANCE_TENURE + COMPLETED_TENURE = CURRENT_TENOR
ORIGNAL_TENOR - CURRENT_TENOR = DIFF_ORIGINAL_CURRENT_TENOR
EMI_OS_AMOUNT - BALANCE_EXCESS = NET_RECEIVABLE
PRE_EMI_DUEAMT - PRE_EMI_OS_AMOUNT = PRE_EMI_RECEIVED_AMT
INTEREST_START_DATE - AUTHORIZATIONDATE = DIFF_AUTH_INT_DATE
CURRENT_INTEREST_RATE_MAX - CURRENT_INTEREST_RATE_MIN = DIFF_CURRENT_INTEREST_RATE_MAX_MIN
MAX_EMI_AMOUNT - MIN_EMI_AMOUNT = DIFF_EMI_AMOUNT_MAX_MIN
ORIGNAL_INTEREST_RATE - CURRENT_INTEREST_RATE = DIFF_ORIGINAL_CURRENT_INTEREST_RATE
BALANCE_TENURE,COMPLETED_TENURE,CURRENT_TENOR,DIFF_ORIGINAL_CURRENT_TENOR,ORIGNAL_TENOR
CHECKING THE DIFFERENT RELATIONSHIP BETWEEN THESE COLUMNS¶
In [13]:
df[['BALANCE_TENURE','COMPLETED_TENURE','CURRENT_TENOR','DIFF_ORIGINAL_CURRENT_TENOR','ORIGNAL_TENOR']].head()
Out[13]:
In [14]:
df[['EMI_OS_AMOUNT','BALANCE_EXCESS','NET_RECEIVABLE']].head()
Out[14]:
In [15]:
df[['EMI_DUEAMT','EMI_OS_AMOUNT','EMI_RECEIVED_AMT']].head()
Out[15]:
In [16]:
df[['DIFF_AUTH_INT_DATE','AUTHORIZATIONDATE','INTEREST_START_DATE']].head()
Out[16]:
In [17]:
df[['MAX_EMI_AMOUNT','MIN_EMI_AMOUNT','DIFF_EMI_AMOUNT_MAX_MIN']].head()
Out[17]:
In [18]:
df[['CURRENT_INTEREST_RATE_MAX','CURRENT_INTEREST_RATE_MIN','DIFF_CURRENT_INTEREST_RATE_MAX_MIN']].head()
Out[18]:
In [19]:
df[['ORIGNAL_INTEREST_RATE','CURRENT_INTEREST_RATE','DIFF_ORIGINAL_CURRENT_INTEREST_RATE']].head()
Out[19]:
In [20]:
df[['BALANCE_EXCESS','EXCESS_AVAILABLE','EXCESS_ADJUSTED_AMT']].head(10)
Out[20]:
In [21]:
(df['COMPLETED_TENURE'] + df['DIFF_ORIGINAL_CURRENT_TENOR']).head()
Out[21]:
In [23]:
df[df['PRE_EMI_DUEAMT'] - df['PRE_EMI_RECEIVED_AMT'] > 0][['PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT','PRE_EMI_RECEIVED_AMT']].head()
Out[23]:
In [24]:
df.shape
Out[24]:
CHECKING AND DROPPING THE NULL VALUES¶
In [25]:
df.isnull().sum()
#Null values are present.NPA_IN_LAST_MONTH & NPA_IN_CURRENT_MONTH are two columns having excess null values. And it would
#be better to drop them off.
Out[25]:
In [26]:
df.duplicated().sum()
#Theres no duplicate records
Out[26]:
In [27]:
df.drop(['NPA_IN_LAST_MONTH','NPA_IN_CURRENT_MONTH'],axis=1,inplace=True)
#Dropping these column as it has lot of null values
In [28]:
df.dropna(inplace=True)
#Dropping the rows having null values in it
In [29]:
df.isnull().sum()
#NO NULL VALUES ARE PRESENT
Out[29]:
In [30]:
df['AGREEMENTID'].nunique()
#We can drop this unique identifier column as this will not be used for our analysis.
Out[30]:
In [31]:
df.shape
Out[31]:
In [32]:
df.drop(['AGREEMENTID'],axis=1,inplace=True)
# Dropping of unique column.
In [33]:
df.head()
Out[33]:
In [34]:
df['FORECLOSURE'].value_counts()
Out[34]:
In [351]:
sns.barplot(x=df['FORECLOSURE'].value_counts().index,y=df['FORECLOSURE'].value_counts())
Out[351]:
In [35]:
df['FORECLOSURE'].value_counts(normalize=True)
Out[35]:
UNIVARIATE AND BIVARIATE ANALYSIS¶
In [36]:
sns.barplot(df.groupby(df['FORECLOSURE'])['BALANCE_EXCESS'].mean().index,df.groupby(df['FORECLOSURE'])['BALANCE_EXCESS'].mean().values)
plt.ylabel('Mean Balance Excess')
#Balance excess is higher in case of foreclosure. As the collaterals are being auctioned to pay off loans
# theres is higher probability of getting the amount greater than the outstanding amount.
Out[36]:
In [37]:
df['Percentage_Completion'] = (df['COMPLETED_TENURE']/df['CURRENT_TENOR']) *100
#Let us add a new column Percentage_Completion
In [38]:
df.head()
Out[38]:
In [39]:
df.describe()
Out[39]:
In [40]:
sns.barplot(df.groupby(df['FORECLOSURE'])['Percentage_Completion'].mean().index,df.groupby(df['FORECLOSURE'])['Percentage_Completion'].mean().values)
plt.ylabel('Mean Percentage Completion')
# This is a interesting insights. Person/Company which are foreclosuring has higher percentage completion rate. Which somehow
# shows us that people are not defaulting willfully but there is some reason due to which this is happening.
Out[40]:
In [353]:
sns.barplot(df.groupby(df['FORECLOSURE'])['BALANCE_TENURE'].mean().index,df.groupby(df['FORECLOSURE'])['BALANCE_TENURE'].mean().values)
plt.ylabel('Mean Balance Tenure')
# Mean completed tenure is more for person getting foreclosed.
Out[353]:
In [42]:
df.groupby(df['FORECLOSURE'])['CURRENT_TENOR'].mean()
Out[42]:
In [43]:
City_counts=pd.DataFrame(df.groupby(['FORECLOSURE','CITY'])['CUSTOMERID'].count().sort_values(ascending=False))
In [44]:
City_counts=City_counts.reset_index()
City_counts[City_counts['FORECLOSURE']==0].head()
Out[44]:
In [45]:
City_counts[City_counts['FORECLOSURE']==1].head()
Out[45]:
In [46]:
sns.barplot(x=df.groupby(['FORECLOSURE'])['COMPLETED_TENURE'].mean().index,y=df.groupby(['FORECLOSURE'])['COMPLETED_TENURE'].mean().values)
plt.xlabel("Completed Tenure")
Out[46]:
In [47]:
sns.barplot(np.array(City_counts[City_counts['FORECLOSURE']==1]['CITY'].head()),City_counts[City_counts['FORECLOSURE']==1]['CUSTOMERID'].head().values)
plt.xlabel('City with most number of FORECLOSURES')
# MUMBAI is the city recording highest number of foreclosure. Followed by Hyderabad. Closure look has to be given on these
#cities.
Out[47]:
In [48]:
sns.barplot(np.array(City_counts[City_counts['FORECLOSURE']==0]['CITY'].head()),City_counts[City_counts['FORECLOSURE']==0]['CUSTOMERID'].head().values)
plt.xlabel('City with most number of non-FORECLOSURES')
# MUMBAI is the city recording highest number of non-foreclosure case as well . Followed by Hyderabad.
Out[48]:
In [49]:
sns.barplot(df['CITY'].value_counts().index[:5],df['CITY'].value_counts().values[:5])
plt.xlabel('Cities with maximum number of loan given')
# MUMBAI AND HYDERABAD gave the highest number of loan.
Out[49]:
In [50]:
PRODUCT_STATUS=pd.DataFrame(df.groupby(['FORECLOSURE','PRODUCT'])['CITY'].count()).reset_index()
In [355]:
sns.barplot(x=PRODUCT_STATUS[PRODUCT_STATUS['FORECLOSURE'] == 0]['PRODUCT'],y=PRODUCT_STATUS[PRODUCT_STATUS['FORECLOSURE'] == 0]['CITY'])
plt.xlabel('Products count for non-foreclosure account')
plt.ylabel(' Counts ')
Out[355]:
In [354]:
sns.barplot(x=PRODUCT_STATUS[PRODUCT_STATUS['FORECLOSURE'] == 1]['PRODUCT'],y=PRODUCT_STATUS[PRODUCT_STATUS['FORECLOSURE'] == 1]['CITY'])
plt.xlabel('Products for foreclosure account')
plt.ylabel(' Counts ')
# Most of the foreclosure is seen for the PRODUCT - HL followed by STHL . A thorough inspection of customer should be
# performed for giving away loans in the category - HL and STHL
Out[354]:
In [356]:
sns.barplot(df['PRODUCT'].value_counts().index,df['PRODUCT'].value_counts().values)
plt.xlabel('Product Codes')
plt.ylabel(' Product Counts ')
#Maximum number of loans has been disbursed as Short Term Home Loan.
Out[356]:
In [54]:
df['MOB'].value_counts(ascending=False).head()
Out[54]:
In [55]:
sns.barplot(df['MOB'].value_counts(ascending=False).head().index,df['MOB'].value_counts(ascending=False).head())
plt.xlabel('Most popular code')
# The most popular code is coming as 9 followed by 7 and 12
Out[55]:
In [56]:
plt.figure(figsize=(10,5))
sns.barplot(df['PRODUCT'], df['LOAN_AMT'],hue=df['FORECLOSURE'])
# The mean value of loan amount for Product LAP is highest. Moreover those person getting foreclosed has highest mean loan
# amount compared to others.
Out[56]:
In [57]:
plt.figure(figsize=(8,5))
sns.barplot(df['PRODUCT'], df['LOAN_AMT'])
Out[57]:
In [58]:
sns.barplot(df['FORECLOSURE'], df['NET_LTV'])
# Net Loan to Value ratio looks same for both the case.
Out[58]:
In [59]:
df['NET_LTV_RANGE']=pd.cut(df['NET_LTV'], [0,10,20,30,40,50,60,70,80,90,100], include_lowest=True)
#Introducing a new column NET_LTV_RANGE to understand under which bracket the NET_LTV is falling . This may help us to
#get some new insights.
In [60]:
df[['NET_LTV','NET_LTV_RANGE']].head()
Out[60]:
In [61]:
plt.figure(figsize=(10,5))
sns.countplot(y=df['NET_LTV_RANGE'])
# We can see that most number of loans has been given to those having NET_LTV_RANGE between 40 to 70.
Out[61]:
In [62]:
plt.figure(figsize=(10,5))
sns.barplot(y=df[df['FORECLOSURE'] == 1]['NET_LTV_RANGE'].value_counts().index , x=df[df['FORECLOSURE'] == 1]['NET_LTV_RANGE'].value_counts().values)
plt.xlabel('Counts corresponding to FORECLOSURE')
# We can see that most number of foreclosure has happened when NET_LTV_RANGE between 70 to 80.
# Bank should try to give loan at safer side in the range od 30 to 50 . As most of the foreclosure is taking place around
# 50-80
Out[62]:
In [63]:
plt.figure(figsize=(10,5))
sns.countplot(y=df['NET_LTV_RANGE'],hue=df['FORECLOSURE'])
plt.xlabel(' Respective counts corresponding to FORECLOSURE & NON-FORECLOSURE')
Out[63]:
In [64]:
df['SCHEMEID'].value_counts().head()
Out[64]:
In [65]:
plt.figure(figsize=(10,5))
sns.barplot(x=df['SCHEMEID'].value_counts().head(5).index , y=df['SCHEMEID'].value_counts().head().values)
plt.xlabel('Most popular scheme')
Out[65]:
In [66]:
plt.figure(figsize=(10,5))
sns.barplot(x=df[df['FORECLOSURE'] == 1]['SCHEMEID'].value_counts(ascending=False).head(5).index , y=df[df['FORECLOSURE'] == 1]['SCHEMEID'].value_counts(ascending=False).head(5).values)
plt.xlabel(' Top 5 Schemes resulting in maximum foreclosure')
Out[66]:
In [67]:
plt.figure(figsize=(10,5))
sns.barplot(x=df[df['FORECLOSURE'] == 0]['SCHEMEID'].value_counts(ascending=False).head(5).index , y=df[df['FORECLOSURE'] == 0]['SCHEMEID'].value_counts(ascending=False).head(5).values)
plt.xlabel('Top 5 Schemes having maximum number of non-foreclosure case')
Out[67]:
In [68]:
df['SCHEMEID'].value_counts(ascending=True).head(48)
#Least popular scheme id
Out[68]:
In [69]:
plt.figure(figsize=(8,5))
sns.barplot(df['MOB'].value_counts(ascending=False).head().index,df['MOB'].value_counts(ascending=False).head())
plt.xlabel('Most popular Internal code')
Out[69]:
In [70]:
plt.figure(figsize=(8,5))
sns.barplot(x=df[df['FORECLOSURE'] == 1]['MOB'].value_counts(ascending=False).head(5).index , y=df[df['FORECLOSURE'] == 1]['MOB'].value_counts(ascending=False).head(5).values)
plt.xlabel('Top 5 Internal code having maximum number of foreclosure case')
Out[70]:
In [71]:
plt.figure(figsize=(8,5))
sns.barplot(x=df[df['FORECLOSURE'] == 0]['MOB'].value_counts(ascending=False).head(5).index , y=df[df['FORECLOSURE'] == 0]['MOB'].value_counts(ascending=False).head(5).values)
plt.xlabel('Top 5 Internal code having maximum number of non-foreclosure case')
Out[71]:
In [72]:
df['FOIR_Range']=pd.cut(df['FOIR'], [0,0.20,0.40,0.60,0.80,1,5,10,15,20,25,30,35], include_lowest=True)
In [73]:
df.isnull().sum()
Out[73]:
In [74]:
#We can clearly see that these records for FOIR column in incorrectly inserted as FOIR cannot be negative and cannot have
#such high values. Hence we will going to drop these 2 rows.
df[df['FOIR_Range'].isnull()][['FOIR','FOIR_Range']]
Out[74]:
In [75]:
df.dropna(inplace=True)
In [76]:
df[df['FOIR_Range'].isnull()][['FOIR','FOIR_Range']]
Out[76]:
In [77]:
df['FOIR_Range'].value_counts()
Out[77]:
In [78]:
plt.figure(figsize=(8,5))
sns.countplot(y=df['FOIR_Range'])
plt.xlabel('Total counts for a specific range ')
Out[78]:
In [79]:
plt.figure(figsize=(8,5))
sns.barplot(y=df[df['FORECLOSURE'] == 1]['FOIR_Range'].value_counts(ascending=False).head(5).index , x=df[df['FORECLOSURE'] == 1]['FOIR_Range'].value_counts(ascending=False).head(5).values)
plt.xlabel('FORECLOSURE counts for a specific range ')
Out[79]:
In [80]:
FOIR_RATIO = (df[df['FORECLOSURE'] == 1]['FOIR_Range'].value_counts())/(df['FOIR_Range'].value_counts())
FOIR_RATIO
#Creating a new feature comprising of the ratio between FORECLOSURE counts in a specific range to the Total counts
# in that range
Out[80]:
In [81]:
plt.figure(figsize=(8,5))
sns.barplot(y=FOIR_RATIO.index,x=FOIR_RATIO.values)
plt.xlabel('Ratio between FORECLOSURE counts in a specific range to the Total counts in that range ')
#We can very well observed that whenever the FOIR ratio exceeded 80 % , the count of foreclosue is drastically increased.
#Moreover in those case when the FOIR ratio is less than 20 percent has also greater count of foreclosure. This may be the
# case of faking oneself income or defaulting willfully and hence the closure look need to be given upen this.
Out[81]:
In [82]:
df.head()
Out[82]:
In [83]:
sns.countplot(x=df[df['FORECLOSURE'] == 0]['CURRENT_INTEREST_RATE_CHANGES'])
plt.xlabel('Effect of CURRENT_INTEREST_RATE_CHANGES on Non-Foreclosure')
Out[83]:
In [84]:
sns.countplot(x=df[df['FORECLOSURE'] == 1]['CURRENT_INTEREST_RATE_CHANGES'])
plt.xlabel('Effect of CURRENT_INTEREST_RATE_CHANGES on Foreclosure')
#Current interest rate changes does not have any effect on foreclosure
Out[84]:
In [85]:
df['Mean_Interest_Rate'] =(df['CURRENT_INTEREST_RATE_MAX'] + df['CURRENT_INTEREST_RATE'] + df['CURRENT_INTEREST_RATE_MIN'])/3
In [86]:
sns.barplot(y=df['Mean_Interest_Rate'],x=df['FORECLOSURE'])
# People getting foreclosed have higher mean interest rate. Although the difference is not that great. But still it can
# have positive impact on increasing case of foreclosure
Out[86]:
In [87]:
df.groupby(['FORECLOSURE'])['Mean_Interest_Rate'].mean()
Out[87]:
In [88]:
sns.barplot(y=df['LOAN_AMT'],x=df['FORECLOSURE'])
# People getting foreclosed have higher loan amount sanctioned. Greater the loan amount
# higher the chance of foreclosure
Out[88]:
In [89]:
sns.barplot(y=df['ORIGNAL_TENOR'],x=df['FORECLOSURE'])
# People having higher original tenor results in higher the chance of foreclosure
Out[89]:
In [90]:
sns.barplot(y=df['COMPLETED_TENURE'],x=df['FORECLOSURE'])
#Person getting foreclosed have larger completed tenure
Out[90]:
In [91]:
sns.barplot(y=df['DPD'],x=df['FORECLOSURE'])
plt.ylabel('Days past due')
#Days past due is an important factor for FORECLOSURE. People getting foreclosed have higher units of DPD
Out[91]:
In [92]:
sns.barplot(y=df['DIFF_ORIGINAL_CURRENT_INTEREST_RATE'],x=df['FORECLOSURE'])
# Difference between Original and Current interest rate is insignificant incase of Foreclosure.
Out[92]:
In [93]:
df.groupby(['FORECLOSURE'])['DIFF_ORIGINAL_CURRENT_INTEREST_RATE'].mean()
Out[93]:
In [94]:
df['MEAN_EMI_AMOUNT'] =(df['EMI_AMOUNT'] + df['MAX_EMI_AMOUNT'] + df['MIN_EMI_AMOUNT'])/3
In [95]:
sns.barplot(y=df['MEAN_EMI_AMOUNT'],x=df['FORECLOSURE'])
# Person getting foreclosed are paying higher mean_emi_amount.
Out[95]:
In [96]:
sns.barplot(y=df['DIFF_CURRENT_INTEREST_RATE_MAX_MIN'],x=df['FORECLOSURE'])
Out[96]:
In [97]:
plt.figure(figsize=(12,8))
sns.scatterplot(x='ORIGNAL_TENOR',y='MEAN_EMI_AMOUNT',data=df,hue='FORECLOSURE')
#We can see that most of the foreclosure case is seen in those case when the Original Tenor is exceeds 180 months of time
# frame . Also in maximum case MEAN_EMI_AMOUNT values lies below 15000000 .
Out[97]:
In [98]:
print(df[df['ORIGNAL_TENOR'] < 180][df['FORECLOSURE'] == 1].shape)
print(df[df['ORIGNAL_TENOR'] < 190][df['FORECLOSURE'] == 1].shape)
In [99]:
df.head()
Out[99]:
In [100]:
plt.figure(figsize=(10,5))
sns.barplot(x=df.groupby(df['AUTHORIZATIONDATE'].dt.year)['NET_DISBURSED_AMT'].sum().index,y=df.groupby(df['AUTHORIZATIONDATE'].dt.year)['NET_DISBURSED_AMT'].sum().values)
plt.xlabel('Sum of Net Disbursed Amount each year')
#THere has been a tremendous increase in disbursed loan amount in 2017 and 2018 .A closer look must be given to find out the
# the reason behind it. Such a drastic increase in loan disbursment may be due to increase in giving substandard loan.
Out[100]:
In [101]:
plt.figure(figsize=(10,5))
sns.barplot(x=df.groupby(df['AUTHORIZATIONDATE'].dt.year)['NET_DISBURSED_AMT'].count().index,y=df.groupby(df['AUTHORIZATIONDATE'].dt.year)['NET_DISBURSED_AMT'].count().values)
plt.xlabel(' Count each year')
# The customer counts after 2016 has increased significantly which may be the reason for increase in value of loan disbursed.
Out[101]:
In [102]:
plt.figure(figsize=(10,5))
sns.barplot(x=df.groupby(df['AUTHORIZATIONDATE'].dt.month)['NET_DISBURSED_AMT'].sum().index,y=df.groupby(df['AUTHORIZATIONDATE'].dt.month)['NET_DISBURSED_AMT'].sum().values)
plt.xlabel('Sum of Net Disbursed Amount each year')
# In the month of March the maximum amount of disbursment has taken place.
Out[102]:
In [103]:
plt.figure(figsize=(10,5))
sns.barplot(x=df.groupby(df['AUTHORIZATIONDATE'].dt.month)['NET_DISBURSED_AMT'].count().index,y=df.groupby(df['AUTHORIZATIONDATE'].dt.month)['NET_DISBURSED_AMT'].count().values)
plt.xlabel('Sum of Net Disbursed Amount each year')
# Customer counts goes similar to the sum of the loan disbursed each month.
Out[103]:
In [104]:
plt.figure(figsize=(10,5))
sns.barplot(x=df[df['FORECLOSURE'] == 1].groupby(df['LAST_RECEIPT_DATE'].dt.year)['FORECLOSURE'].count().index,y=df[df['FORECLOSURE'] == 1].groupby(df['LAST_RECEIPT_DATE'].dt.year)['FORECLOSURE'].count().values)
plt.xlabel(' Year performance in case of Forelosure')
# Year 2017 has seen the most number of foreclosure. If we assume this dataset belongs to the companies from India ,
# demonitisation may have played a huge role in increase in foreclosure.
Out[104]:
In [105]:
plt.figure(figsize=(10,5))
sns.barplot(x=df[df['FORECLOSURE'] == 1].groupby(df['LAST_RECEIPT_DATE'].dt.month)['FORECLOSURE'].count().index,y=df[df['FORECLOSURE'] == 1].groupby(df['LAST_RECEIPT_DATE'].dt.month)['FORECLOSURE'].count().values)
plt.xlabel(' Year performance in case of Forelosure')
# No visible pattern is seen . Nothing can be inferred in this case.
Out[105]:
In [106]:
plt.figure(figsize=(8,5))
sns.barplot(x=df[df['FORECLOSURE'] == 1].groupby(df['LATEST_TRANSACTION_MONTH'])['FORECLOSURE'].count().index,y=df[df['FORECLOSURE'] == 1].groupby(df['LATEST_TRANSACTION_MONTH'])['FORECLOSURE'].count().values)
plt.xlabel(' LATEST TRANSACTION MONTH in case of Forelosure')
Out[106]:
In [107]:
plt.figure(figsize=(8,5))
sns.barplot(x=df[df['FORECLOSURE'] == 0].groupby(df['LATEST_TRANSACTION_MONTH'])['FORECLOSURE'].count().index,y=df[df['FORECLOSURE'] == 0].groupby(df['LATEST_TRANSACTION_MONTH'])['FORECLOSURE'].count().values)
plt.xlabel(' LATEST TRANSACTION MONTH in case of Non - Forelosure')
Out[107]:
In [108]:
plt.figure(figsize=(8,5))
sns.countplot(df['LATEST_TRANSACTION_MONTH'] , hue=df['FORECLOSURE'])
plt.xlabel(' LATEST TRANSACTION MONTH in case of Forelosure')
Out[108]:
In [109]:
df.head()
Out[109]:
Removal of unwanted variables¶
In [110]:
df_1=pd.concat([df,pd.get_dummies(df['PRODUCT'])],axis=1)
#One hot encoding for Product Column
In [111]:
df_1.head(5)
Out[111]:
In [112]:
df_1.shape
Out[112]:
In [113]:
df_1.drop(['PRODUCT'],axis=1,inplace=True)
# We can drop this PRODUCT column as it has already been taken care in form of One hot encoding
In [114]:
df[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE', 'CURRENT_INTEREST_RATE', \
'CURRENT_INTEREST_RATE_MAX', 'CURRENT_INTEREST_RATE_MIN','CURRENT_INTEREST_RATE_CHANGES', 'CURRENT_TENOR',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN', 'DIFF_EMI_AMOUNT_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT','EMI_RECEIVED_AMT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'LOAN_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_DISBURSED_AMT', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS','ORIGNAL_INTEREST_RATE', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PAID_INTEREST', 'PAID_PRINCIPAL', 'PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT', 'PRE_EMI_RECEIVED_AMT','Percentage_Completion',\
'Mean_Interest_Rate','MEAN_EMI_AMOUNT' ]].corr()
Out[114]:
In [115]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE', 'CURRENT_INTEREST_RATE', \
'CURRENT_INTEREST_RATE_MAX', 'CURRENT_INTEREST_RATE_MIN','CURRENT_INTEREST_RATE_CHANGES', 'CURRENT_TENOR',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN', 'DIFF_EMI_AMOUNT_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT','EMI_RECEIVED_AMT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'LOAN_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_DISBURSED_AMT', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS','ORIGNAL_INTEREST_RATE', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PAID_INTEREST', 'PAID_PRINCIPAL', 'PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT', 'PRE_EMI_RECEIVED_AMT','Percentage_Completion',\
'Mean_Interest_Rate','MEAN_EMI_AMOUNT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
# Multicollinearity is present in lot of variables . Hence we will have to intelligently drop some of the highly correlated
# variables.
Out[115]:
In [116]:
df_1.head()
Out[116]:
Establishing relationship between different columns . Below are the columns which are derived from the other columns. This information will help us in performing feature engineering.¶
PRE_EMI_DUEAMT= PRE_EMI_RECEIVED_AMT+PRE_EMI_OS_AMOUNT
OUTSTANDING_PRINCIPAL+ PAID_PRINCIPAL = LOAN_AMT
BALANCE_TENURE + COMPLETED_TENURE = CURRENT_TENOR
ORIGNAL_TENOR - CURRENT_TENOR = DIFF_ORIGINAL_CURRENT_TENOR
EMI_OS_AMOUNT - BALANCE_EXCESS = NET_RECEIVABLE
PRE_EMI_DUEAMT - PRE_EMI_OS_AMOUNT = PRE_EMI_RECEIVED_AMT
INTEREST_START_DATE - AUTHORIZATIONDATE = DIFF_AUTH_INT_DATE
CURRENT_INTEREST_RATE_MAX - CURRENT_INTEREST_RATE_MIN = DIFF_CURRENT_INTEREST_RATE_MAX_MIN
MAX_EMI_AMOUNT - MIN_EMI_AMOUNT = DIFF_EMI_AMOUNT_MAX_MIN
ORIGNAL_INTEREST_RATE - CURRENT_INTEREST_RATE = DIFF_ORIGINAL_CURRENT_INTEREST_RATE
BALANCE_TENURE,COMPLETED_TENURE,CURRENT_TENOR,DIFF_ORIGINAL_CURRENT_TENOR,ORIGNAL_TENOR
Mean_Interest_Rate = CURRENT_INTEREST_RATE_MAX + CURRENT_INTEREST_RATE + CURRENT_INTEREST_RATE_MIN
In [117]:
# As Mean_Interest_Rate column is derived from CURRENT_INTEREST_RATE_MAX,CURRENT_INTEREST_RATE,CURRENT_INTEREST_RATE_MIN
# column , we can drop these 3 columns with respect to feature engineering.
In [118]:
df_1.drop(['CURRENT_INTEREST_RATE','CURRENT_INTEREST_RATE_MAX','CURRENT_INTEREST_RATE_MIN'],axis=1,inplace=True)
In [119]:
df_1.head()
Out[119]:
In [120]:
df[['BALANCE_TENURE','COMPLETED_TENURE','CURRENT_TENOR','ORIGNAL_TENOR']].head()
# We can see that BALANCE_TENURE,COMPLETED_TENURE,CURRENT_TENOR are related with each other. Moreover Balance Tenur
# and Current Tenore are highly correlated with each other. Hence we will going to drop 1 column .Here I am dropping Current
# TENOR
Out[120]:
In [121]:
df_1.drop(['CURRENT_TENOR'],axis=1,inplace=True)
In [122]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df_1[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE','CURRENT_INTEREST_RATE_CHANGES','Mean_Interest_Rate',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN', 'DIFF_EMI_AMOUNT_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT','EMI_RECEIVED_AMT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'LOAN_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_DISBURSED_AMT', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS','ORIGNAL_INTEREST_RATE', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PAID_INTEREST', 'PAID_PRINCIPAL', 'PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT', 'PRE_EMI_RECEIVED_AMT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
Out[122]:
In [123]:
df_1.drop(['ORIGNAL_INTEREST_RATE'],axis=1,inplace=True)
#As the ORIGNAL_INTEREST_RATE is nicely captured by Mean_Interest_Rate , hence we will be dropping ORIGNAL_INTEREST_RATE.
In [124]:
df_1.drop(['DIFF_EMI_AMOUNT_MAX_MIN'],axis=1,inplace=True)
# DIFF_EMI_AMOUNT_MAX_MIN is highly correlated with MAX_EMI_AMOUNT.Hence we will be dropping DIFF_EMI_AMOUNT_MAX_MIN column.
In [125]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df_1[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE','CURRENT_INTEREST_RATE_CHANGES','Mean_Interest_Rate',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT','EMI_RECEIVED_AMT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'LOAN_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_DISBURSED_AMT', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PAID_INTEREST', 'PAID_PRINCIPAL', 'PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
Out[125]:
In [126]:
df_1[(df_1['PRE_EMI_DUEAMT'] - df_1['PRE_EMI_RECEIVED_AMT'])>0][['PRE_EMI_DUEAMT','PRE_EMI_RECEIVED_AMT']].shape
Out[126]:
In [127]:
df_1.drop(['EMI_RECEIVED_AMT'],axis=1,inplace=True)
# PRE_EMI_DUEAMT is somewhat equal to PRE_EMI_RECEIVED_AMT .So both the variables are highly correlated. Moreover from
# business point of view if PRE_EMI_RECEIVED_AMT should be same as that of PRE_EMI_DUEAMT.If not person may start defaulting
# Hence we will going to drop EMI_RECEIVED_AMT column
In [128]:
df_1.drop(['PRE_EMI_RECEIVED_AMT'],axis=1,inplace=True)
# Similarly PRE_EMI_RECEIVED_AMT is highly correlated PRE_EMI_DUEAMT . Hence we will be dropping PRE_EMI_DUEAMT column
In [129]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df_1[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE','CURRENT_INTEREST_RATE_CHANGES','Mean_Interest_Rate',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'LOAN_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_DISBURSED_AMT', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PAID_INTEREST', 'PAID_PRINCIPAL', 'PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
Out[129]:
In [130]:
df_1.head()
Out[130]:
In [131]:
sns.countplot(df_1['LAP'])
Out[131]:
In [132]:
sns.countplot(df_1['STHL'])
Out[132]:
In [133]:
df_1['PAID_AMOUNT'] =df_1['PAID_PRINCIPAL'] + df_1['PAID_INTEREST']
# We can add PAID_PRINCIPAL and PAID_INTEREST column to get new column ,PAID_AMOUNT. This can help us to drop first 2 column
# used.
Establishing relationship between different columns . Below are the columns which are derived from the other columns. This information will help us in performing feature engineering.¶
PRE_EMI_DUEAMT= PRE_EMI_RECEIVED_AMT ## +PRE_EMI_OS_AMOUNT
OUTSTANDING_PRINCIPAL+ PAID_PRINCIPAL ## = LOAN_AMT
BALANCE_TENURE + COMPLETED_TENURE = ## CURRENT_TENOR
ORIGNAL_TENOR - CURRENT_TENOR = DIFF_ORIGINAL_CURRENT_TENOR
EMI_OS_AMOUNT - BALANCE_EXCESS = NET_RECEIVABLE
PRE_EMI_DUEAMT - PRE_EMI_OS_AMOUNT = PRE_EMI_RECEIVED_AMT
INTEREST_START_DATE - AUTHORIZATIONDATE = DIFF_AUTH_INT_DATE
CURRENT_INTEREST_RATE_MAX - CURRENT_INTEREST_RATE_MIN = DIFF_CURRENT_INTEREST_RATE_MAX_MIN
MAX_EMI_AMOUNT - MIN_EMI_AMOUNT = ##DIFF_EMI_AMOUNT_MAX_MIN
ORIGNAL_INTEREST_RATE ## - CURRENT_INTEREST_RATE = DIFF_ORIGINAL_CURRENT_INTEREST_RATE
BALANCE_TENURE,COMPLETED_TENURE,CURRENT_TENOR,DIFF_ORIGINAL_CURRENT_TENOR,ORIGNAL_TENOR
Mean_Interest_Rate = #CURRENT_INTEREST_RATE_MAX + CURRENT_INTEREST_RATE + CURRENT_INTEREST_RATE_MIN
'CURRENT_INTEREST_RATE','CURRENT_INTEREST_RATE_MAX','CURRENT_INTEREST_RATE_MIN'
In [134]:
df_1.drop(['PAID_PRINCIPAL','PAID_INTEREST'],axis=1,inplace=True)
In [135]:
df[df['LOAN_AMT'] -df['NET_DISBURSED_AMT'] > 0][['LOAN_AMT','NET_DISBURSED_AMT']].shape
Out[135]:
In [136]:
df_1.drop(['LOAN_AMT'],axis=1,inplace=True) #Net disbursed amount is somewhat same as that of loan_amt . As we know the
# all the interest calculation is done on the NET_DISBURSED_AMT and not on LOAN_AMOUNT . Hence for calculation and model
# development purpose , we will keep NET_DISBURSED_AMT and drop LOAN_AMT
In [137]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df_1[['BALANCE_EXCESS', 'BALANCE_TENURE','COMPLETED_TENURE','CURRENT_INTEREST_RATE_CHANGES','Mean_Interest_Rate',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', \
'LATEST_TRANSACTION_MONTH', 'NET_DISBURSED_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'MONTHOPENING', 'NET_LTV',
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS', 'ORIGNAL_TENOR',\
'OUTSTANDING_PRINCIPAL','PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT','PAID_AMOUNT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
# Still there are multicollinearity present for certain variables.
Out[137]:
In [138]:
df_2=df_1.copy()
# Taking a copy of df_1
In [139]:
df_2.drop(['MONTHOPENING'],axis=1,inplace=True)
# MONTHOPENING is not usefull for our analysis. Hence we will be dropping it
In [140]:
plt.figure(figsize=(20,20))
sns.heatmap(data=df_2[['BALANCE_EXCESS','BALANCE_TENURE','COMPLETED_TENURE','CURRENT_INTEREST_RATE_CHANGES','Mean_Interest_Rate',\
'DIFF_AUTH_INT_DATE','DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'DIFF_ORIGINAL_CURRENT_TENOR','DPD', 'DUEDAY', 'EMI_AMOUNT', 'EMI_DUEAMT', 'EMI_OS_AMOUNT',\
'EXCESS_ADJUSTED_AMT', 'EXCESS_AVAILABLE', 'FOIR','LAST_RECEIPT_AMOUNT', 'OUTSTANDING_PRINCIPAL',\
'LATEST_TRANSACTION_MONTH', 'NET_DISBURSED_AMT', 'MAX_EMI_AMOUNT','MIN_EMI_AMOUNT', 'NET_LTV',\
'NET_RECEIVABLE', 'NUM_EMI_CHANGES', 'NUM_LOW_FREQ_TRANSACTIONS', 'ORIGNAL_TENOR','PRE_EMI_DUEAMT','PRE_EMI_OS_AMOUNT','PAID_AMOUNT']].corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
# There are still certain variables which we will have to drop as multicollinearity is still present
Out[140]:
In [141]:
# PRE_EMI_OS_AMOUNT 6.928615e+14
# EMI_RECEIVED_AMT 2.144571e+14
In [142]:
df_2.shape
Out[142]:
Checking of multicollinearity with VIF¶
In [143]:
from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.tools.tools import add_constant
In [144]:
df_2=add_constant(df_2)
In [145]:
df_2.head(5)
Out[145]:
In [146]:
df_2[df_2['LAST_RECEIPT_DATE'] < df_2['AUTHORIZATIONDATE']].head()[['AUTHORIZATIONDATE','INTEREST_START_DATE','LAST_RECEIPT_DATE']]
Out[146]:
In [147]:
# As per business understanding , LAST_RECEIPT_DATE should be greater than AUTHORIZATIONDATE & INTEREST_START_DATE.
# and INTEREST_START_DATE should be greater than or equal to AUTHORIZATIONDATE.
#Whenever INTEREST_START_DATE is less than AUTHORIZATIONDATE we will going to replace INTEREST_START_DATE with
# AUTHORIZATIONDATE
#Whenever LAST_RECEIPT_DATE is less than INTEREST_START_DATE we will going to replace LAST_RECEIPT_DATE with
# INTEREST_START_DATE
In [148]:
df_2['LAST_RECEIPT_DATE'] = np.where(df_2['LAST_RECEIPT_DATE'] < df_2['INTEREST_START_DATE'],\
df_2['INTEREST_START_DATE'],df_2['LAST_RECEIPT_DATE'])
df_2['INTEREST_START_DATE'] = np.where(df_2['INTEREST_START_DATE'] < df_2['AUTHORIZATIONDATE'],\
df_2['AUTHORIZATIONDATE'],df_2['INTEREST_START_DATE'])
In [149]:
df_2[df_2['LAST_RECEIPT_DATE'] < df_2['INTEREST_START_DATE']].head()
Out[149]:
In [150]:
df_2['DIFF_LAST_RECEIPT_INT_DATE'] = df_2['LAST_RECEIPT_DATE'] - df_2['INTEREST_START_DATE']
# Addition of new column DIFF_LAST_RECEIPT_INT_DATE which takes difference between LAST_RECEIPT_DATE & INTEREST_START_DATE
In [151]:
df_2['DIFF_LAST_RECEIPT_INT_DATE']=df_2['DIFF_LAST_RECEIPT_INT_DATE'].dt.days
In [152]:
df_2.drop(['AUTHORIZATIONDATE','DIFF_AUTH_INT_DATE','CITY','CUSTOMERID','INTEREST_START_DATE',\
'LAST_RECEIPT_DATE','SCHEMEID','MOB'],axis=1,inplace=True)
# INTEREST_START_DATE,LAST_RECEIPT_DATE has been taken care inform of DIFF_LAST_RECEIPT_INT_DATE column
# AUTHORIZATIONDATE and INTEREST_START_DATE is somewhat same as each other. They will not be used for model development.
# Hence we will going to drop them.
# DIFF_AUTH_INT_DATE will not be useful for understanding. Hence we will going to drop them.
# We will going to drop CITY , SCHEMEID , MOB column as well for model development purpose as they doesn't seems to be
# important for model building.
In [153]:
df_2.drop(['EMI_AMOUNT','MAX_EMI_AMOUNT','MIN_EMI_AMOUNT'],axis=1,inplace=True)
#All these 3 columns have been combined to form a new column Mean_emi_amount .
#(df['MEAN_EMI_AMOUNT'] =(df['EMI_AMOUNT'] + df['MAX_EMI_AMOUNT'] + df['MIN_EMI_AMOUNT'])/3 )
#Hence we will be dropping these 3 columns
In [154]:
df_2.info()
In [155]:
df_2.drop(['NET_LTV_RANGE','FOIR_Range'],axis=1,inplace=True)
VIF CALCULATION¶
In [156]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
# The VIF values are certain columns are very high . We will try to intelligenty feature engineer or drop the columns
# to pull back the VIF within limits
Out[156]:
In [157]:
df_2.drop(['PRE_EMI_OS_AMOUNT'],axis=1,inplace=True)
# As VIF values is highest for this column hence we will going to drop it. and by using other columns HL,LAP,STLAP
# we can easily determine the status of STHL column. Hence we will be dropping STHL column
In [158]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[158]:
In [159]:
df_2.drop(['HL'],axis=1,inplace=True)
# PRE_EMI_OS_AMOUNT 6.928615e+14
# EMI_RECEIVED_AMT 2.144571e+14
# Looking at the heatmap , we can observe that EMI_RECEIVED_AMT is highly correlated with other variables . But this
# correlation is not seen for PRE_EMI_OS_AMOUNT . Hence we will be dropping EMI_RECEIVED_AMT rather than PRE_EMI_OS_AMOUNT
In [160]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[160]:
Establishing relationship between different columns . Below are the columns which are derived from the other columns. This information¶
will help us in performing feature engineering.
PRE_EMI_DUEAMT= PRE_EMI_RECEIVED_AMT ## +PRE_EMI_OS_AMOUNT
OUTSTANDING_PRINCIPAL+ PAID_PRINCIPAL ## = LOAN_AMT
BALANCE_TENURE + COMPLETED_TENURE = ## CURRENT_TENOR
ORIGNAL_TENOR - CURRENT_TENOR = DIFF_ORIGINAL_CURRENT_TENOR
EMI_OS_AMOUNT - BALANCE_EXCESS = NET_RECEIVABLE
PRE_EMI_DUEAMT - PRE_EMI_OS_AMOUNT = PRE_EMI_RECEIVED_AMT
INTEREST_START_DATE - AUTHORIZATIONDATE = DIFF_AUTH_INT_DATE
CURRENT_INTEREST_RATE_MAX - ##CURRENT_INTEREST_RATE_MIN =
DIFF_CURRENT_INTEREST_RATE_MAX_MIN
MAX_EMI_AMOUNT - MIN_EMI_AMOUNT = ##DIFF_EMI_AMOUNT_MAX_MIN
ORIGNAL_INTEREST_RATE ## - CURRENT_INTEREST_RATE = DIFF_ORIGINAL_CURRENT_INTEREST_RATE
BALANCE_TENURE,COMPLETED_TENURE,CURRENT_TENOR,DIFF_ORIGINAL_CURRENT_TENOR,ORIGNAL_TENOR
Mean_Interest_Rate = #CURRENT_INTEREST_RATE_MAX + CURRENT_INTEREST_RATE + CURRENT_INTEREST_RATE_MIN
'CURRENT_INTEREST_RATE','CURRENT_INTEREST_RATE_MAX','CURRENT_INTEREST_RATE_MIN'
In [162]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[162]:
In [163]:
df_2.drop(['EXCESS_AVAILABLE'],axis=1,inplace=True)
# We will be dropping EXCESS_AVAILABLE as it is has high correlation with other variable and has infinity VIF values.
In [164]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[164]:
In [165]:
df_2.drop(['DIFF_ORIGINAL_CURRENT_TENOR'],axis=1,inplace=True)
In [166]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[166]:
In [167]:
df_2.drop(['BALANCE_EXCESS'],axis=1,inplace=True)
In [168]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[168]:
In [169]:
df_2.drop(['NET_DISBURSED_AMT'],axis=1,inplace=True)
In [170]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[170]:
In [171]:
df_2.drop(['DIFF_LAST_RECEIPT_INT_DATE'],axis=1,inplace=True)
In [172]:
pd.Series([variance_inflation_factor(df_2.values, i)
for i in range(df_2.shape[1])],
index=df_2.columns)
Out[172]:
In [173]:
plt.figure(figsize=(20,20))
sns.heatmap(df_2.corr(),annot=True,fmt='0.1g',linewidths=2, linecolor='black',cbar_kws= {'orientation': 'horizontal'})
Out[173]:
In [174]:
df_2.head()
Out[174]:
Train Test Split¶
In [175]:
df4=df_2.copy()
In [176]:
from sklearn.model_selection import train_test_split
In [177]:
x=df4.drop(['FORECLOSURE'],axis=1)
In [178]:
y=df4.pop('FORECLOSURE')
In [179]:
x_new=x.copy()
y_new=y.copy()
In [180]:
x_train_new,x_test_new,y_train_new,y_test_new=train_test_split(x_new,y_new,test_size=0.3,random_state=0)
In [181]:
x_new_copy=pd.DataFrame()
x_new_copy['LAP']=x_train_new['LAP']
x_new_copy['STHL']=x_train_new['STHL']
x_new_copy['STLAP']=x_train_new['STLAP']
In [182]:
x_new_copy.head(2)
Out[182]:
In [183]:
x_new_copy.reset_index(drop=True,inplace=True)
Z score Transformation¶
In [184]:
from scipy.stats import zscore
In [185]:
from sklearn.preprocessing import StandardScaler
In [186]:
x_train_new.drop(['const','LAP','STHL','STLAP'],axis=1,inplace=True)
In [187]:
x_test_new.drop(['const'],axis=1,inplace=True)
In [188]:
x_train_new.shape
Out[188]:
In [189]:
from sklearn.preprocessing import StandardScaler
ss = StandardScaler()
In [190]:
x_train_new=pd.DataFrame(ss.fit_transform(x_train_new),columns = x_train_new.columns)
In [191]:
x_train_new.head()
Out[191]:
In [192]:
x_test_new.head()
Out[192]:
In [193]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['BALANCE_TENURE'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['DPD'],ax=ax[0,1])
sns.distplot(x_train_new['COMPLETED_TENURE'],ax=ax[0,2])
sns.distplot(x_train_new['CURRENT_INTEREST_RATE_CHANGES'],ax=ax[1,0])
sns.distplot(x_train_new['DIFF_CURRENT_INTEREST_RATE_MAX_MIN'],ax=ax[1,1])
sns.distplot(x_train_new['DIFF_ORIGINAL_CURRENT_INTEREST_RATE'],ax=ax[1,2])
Out[193]:
In [194]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['DUEDAY'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['EMI_DUEAMT'],ax=ax[0,1])
sns.distplot(x_train_new['EMI_OS_AMOUNT'],ax=ax[0,2])
sns.distplot(x_train_new['EXCESS_ADJUSTED_AMT'],ax=ax[1,0])
sns.distplot(x_train_new['FOIR'],ax=ax[1,1])
sns.distplot(x_train_new['LAST_RECEIPT_AMOUNT'],ax=ax[1,2])
Out[194]:
In [195]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(df4['LATEST_TRANSACTION_MONTH'],ax=ax[0,0],kde=True)
sns.distplot(df4['NET_LTV'],ax=ax[0,1])
sns.distplot(df4['NET_RECEIVABLE'],ax=ax[0,2])
sns.distplot(df4['NUM_EMI_CHANGES'],ax=ax[1,0])
sns.distplot(df4['NUM_LOW_FREQ_TRANSACTIONS'],ax=ax[1,1])
sns.distplot(df4['ORIGNAL_TENOR'],ax=ax[1,2])
Out[195]:
In [196]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['OUTSTANDING_PRINCIPAL'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['PRE_EMI_DUEAMT'],ax=ax[0,1])
sns.distplot(x_train_new['Percentage_Completion'],ax=ax[0,2])
sns.distplot(x_train_new['Mean_Interest_Rate'],ax=ax[1,0])
sns.distplot(x_train_new['MEAN_EMI_AMOUNT'],ax=ax[1,1])
sns.distplot(x_train_new['PAID_AMOUNT'],ax=ax[1,2])
Out[196]:
In [197]:
for col in x_train_new.columns:
print("{} column has {} records having zscore greater than 3".format(col,len(x_train_new[x_train_new[col]>3])))
print("{} column has {} records having zscore lesser than -3".format(col,len(x_train_new[x_train_new[col]<-3])))
print("")
In [198]:
for col in x_train_new.columns:
x_train_new[col]=np.where(x_train_new[col] >= 3,3,x_train_new[col])
x_train_new[col]=np.where(x_train_new[col] <= -3,-3,x_train_new[col])
In [199]:
for col in x_train_new.columns:
print("{} column has {} records having zscore greater than 3".format(col,len(x_train_new[x_train_new[col]>3])))
print("{} column has {} records having zscore lesser than -3".format(col,len(x_train_new[x_train_new[col]<-3])))
print("")
In [200]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['BALANCE_TENURE'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['DPD'],ax=ax[0,1])
sns.distplot(x_train_new['COMPLETED_TENURE'],ax=ax[0,2])
sns.distplot(x_train_new['CURRENT_INTEREST_RATE_CHANGES'],ax=ax[1,0])
sns.distplot(x_train_new['DIFF_CURRENT_INTEREST_RATE_MAX_MIN'],ax=ax[1,1])
sns.distplot(x_train_new['DIFF_ORIGINAL_CURRENT_INTEREST_RATE'],ax=ax[1,2])
Out[200]:
In [201]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['DUEDAY'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['EMI_DUEAMT'],ax=ax[0,1])
sns.distplot(x_train_new['EMI_OS_AMOUNT'],ax=ax[0,2])
sns.distplot(x_train_new['EXCESS_ADJUSTED_AMT'],ax=ax[1,0])
sns.distplot(x_train_new['FOIR'],ax=ax[1,1])
sns.distplot(x_train_new['LAST_RECEIPT_AMOUNT'],ax=ax[1,2])
Out[201]:
In [202]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['LATEST_TRANSACTION_MONTH'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['NET_LTV'],ax=ax[0,1])
sns.distplot(x_train_new['NET_RECEIVABLE'],ax=ax[0,2])
sns.distplot(x_train_new['NUM_EMI_CHANGES'],ax=ax[1,0])
sns.distplot(x_train_new['NUM_LOW_FREQ_TRANSACTIONS'],ax=ax[1,1])
sns.distplot(x_train_new['ORIGNAL_TENOR'],ax=ax[1,2])
Out[202]:
In [203]:
f,ax = plt.subplots(2,3,figsize=(18,10),squeeze=True)
sns.distplot(x_train_new['OUTSTANDING_PRINCIPAL'],ax=ax[0,0],kde=True)
sns.distplot(x_train_new['PRE_EMI_DUEAMT'],ax=ax[0,1])
sns.distplot(x_train_new['Percentage_Completion'],ax=ax[0,2])
sns.distplot(x_train_new['Mean_Interest_Rate'],ax=ax[1,0])
sns.distplot(x_train_new['MEAN_EMI_AMOUNT'],ax=ax[1,1])
sns.distplot(x_train_new['PAID_AMOUNT'],ax=ax[1,2])
Out[203]:
In [204]:
x_train_new.head()
Out[204]:
Z scaling Test Data¶
In [205]:
df_test=x_test_new.copy()
In [206]:
df_test.head(2)
Out[206]:
In [207]:
df_test1=pd.DataFrame()
df_test1['LAP']=x_test_new['LAP']
df_test1['STHL']=x_test_new['STHL']
df_test1['STLAP']=x_test_new['STLAP']
In [208]:
df_test1.head(2)
Out[208]:
In [209]:
df_test1.reset_index(drop=True,inplace=True)
In [210]:
x_test_new.drop(['LAP','STHL','STLAP'],axis=1,inplace=True)
In [211]:
x_test_new=pd.DataFrame(ss.transform(x_test_new),columns = x_test_new.columns)
In [212]:
x_test_new.head(10)
Out[212]:
In [213]:
x_test_new=pd.concat([x_test_new,df_test1],axis=1)
x_test_new.head(2)
Out[213]:
In [214]:
x_train_new= pd.concat([x_train_new,x_new_copy],axis=1)
In [215]:
x_test_new.head(2)
Out[215]:
In [216]:
x_train_new.head(2)
Out[216]:
Clustering Analysis¶
KMEANS¶
In [217]:
from sklearn.cluster import KMeans
In [218]:
df_clusters=df_2.copy()
In [219]:
df_clusters.head()
Out[219]:
In [220]:
df_clusters.drop(['const'],axis=1,inplace=True)
In [221]:
nbfc_arr=np.array(df_clusters)
In [222]:
wss=[]
for i in range(1,10):
KM = KMeans(n_clusters=i)
KM.fit(nbfc_arr)
wss.append(KM.inertia_)
In [223]:
plt.plot(range(1,10), wss)
Out[223]:
In [224]:
k_means=KMeans(n_clusters=3)
k_means.fit(df_clusters)
labels=k_means.labels_
# Taking 3 clusters to take into account.
In [225]:
from sklearn.metrics import silhouette_samples,silhouette_score
In [226]:
silhouette_score(df_clusters,labels)
Out[226]:
In [227]:
df_clusters['silhouette_samples']=silhouette_samples(df_clusters,labels)
In [228]:
df_clusters['Kmeans_Cluster'] = labels
In [229]:
df_clusters.head()
Out[229]:
In [230]:
df_clusters['Kmeans_Cluster'].value_counts()
Out[230]:
In [231]:
df_clusters['Customer_ID'] = df_1['CUSTOMERID']
In [232]:
df_clusters[df_clusters['Kmeans_Cluster'] == 0].describe()
Out[232]:
In [233]:
df_clusters[df_clusters['Kmeans_Cluster'] == 2].describe()
Out[233]:
In [234]:
# Customers belongs to cluster 2 has better financials. They are those customers which has lesser Days past dues ,Due Day,
# EMI_DUEAMT ,EMI_OS_AMOUNT,FOIR,NET_LTV,OUTSTANDING_PRINCIPAL . They are the ones which has lesser probabilites to
# get foreclosed
# Customers belongs to cluster 0 has poor financials. They are those customers which has more number of Days past dues ,
# Due Day, EMI_DUEAMT ,EMI_OS_AMOUNT,FOIR,NET_LTV,OUTSTANDING_PRINCIPAL .They are the ones which has higher probabilites to
# get foreclosed. Hence NBFC should give high focus to these customers
Machine Learning¶
Logistic Regression¶
In [235]:
from sklearn.linear_model import LogisticRegressionCV,LogisticRegression
In [236]:
log_modelCV=LogisticRegressionCV()
In [237]:
log_modelCV=log_modelCV.fit(x_train_new,y_train_new)
In [238]:
log_modelCV.score(x_train_new,y_train_new)
Out[238]:
In [239]:
log_modelCV.score(x_test_new,y_test_new)
Out[239]:
In [240]:
log_model=LogisticRegression()
log_model=log_model.fit(x_train_new,y_train_new)
In [241]:
log_model.score(x_train_new,y_train_new)
Out[241]:
In [242]:
log_model.score(x_test_new,y_test_new)
Out[242]:
In [243]:
from sklearn.metrics import classification_report,confusion_matrix
In [244]:
def confusionmatrix(y_actual,y_predict):
sns.heatmap(confusion_matrix(y_actual,y_predict),annot=True,fmt='g')
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()
In [245]:
def class_report(y_actual,y_predict):
print(classification_report(y_actual,y_predict))
In [246]:
confusionmatrix(y_train_new,log_model.predict(x_train_new))
In [247]:
class_report(y_train_new,log_model.predict(x_train_new))
In [248]:
confusionmatrix(y_test_new,log_model.predict(x_test_new))
In [249]:
class_report(y_test_new,log_model.predict(x_test_new))
In [250]:
confusionmatrix(y_train_new,log_modelCV.predict(x_train_new))
class_report(y_train_new,log_modelCV.predict(x_train_new))
confusionmatrix(y_test_new,log_modelCV.predict(x_test_new))
class_report(y_test_new,log_modelCV.predict(x_test_new))
In [251]:
y_train_new.value_counts()
Out[251]:
In [252]:
import imblearn
In [253]:
from imblearn.over_sampling import SMOTE
In [254]:
sm = SMOTE(random_state=0)
x_train_smote, y_train_smote = sm.fit_resample(x_train_new, y_train_new)
In [255]:
x_train.info()
In [256]:
y_train_smote.value_counts()
Out[256]:
In [257]:
log_modelCV_smote=LogisticRegressionCV()
log_modelCV_smote=log_modelCV_smote.fit(x_train_smote,y_train_smote)
print(log_modelCV_smote.score(x_train_smote,y_train_smote))
In [258]:
print(log_modelCV_smote.score(x_test_new,y_test_new))
In [259]:
class_report(y_train_new,log_modelCV_smote.predict(x_train_new))
In [260]:
class_report(y_test_new,log_modelCV_smote.predict(x_test_new))
In [261]:
confusionmatrix(y_train_new,log_modelCV_smote.predict(x_train_new))
In [262]:
confusionmatrix(y_test_new,log_modelCV_smote.predict(x_test_new))
Defining a function to plot roc_curve and calculate roc_auc_score¶
In [263]:
from sklearn.metrics import roc_auc_score,roc_curve
In [264]:
def roc_plot(model_name,x_data,y_lables):
probs1=model_name.predict_proba(x_data)
probs1=probs1[:,1]
print('roc_auc_score -->',roc_auc_score(y_lables,probs1))
fpr,tpr,thresholds=roc_curve(y_lables,probs1)
plt.plot([0,1],[0,1],linestyle='--')
plt.plot(fpr,tpr,marker='.')
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC')
plt.show()
#model_name is the name of the model, x_data is the data that needs to be passed and y_labels is the class values
In [265]:
#Graphical representation of the contrast between true positive rates and false-positive rates at various thresholds.
In [266]:
roc_plot(log_modelCV_smote,x_train_new,y_train_new)
In [267]:
roc_plot(log_modelCV_smote,x_test_new,y_test_new)
In [268]:
log_modelCV_smote.coef_[0]
Out[268]:
In [269]:
plt.figure(figsize=(10,8))
sns.barplot(x=log_modelCV_smote.coef_[0] ,y=x_test_new.columns)
Out[269]:
In [270]:
roc_plot(log_modelCV,x_train_new,y_train_new)
roc_plot(log_modelCV,x_test_new,y_test_new)
Support Vector Machine¶
In [271]:
from sklearn.svm import SVC
In [272]:
svc_smote=SVC(probability=True)
In [273]:
svc_smote=svc_smote.fit(x_train_smote,y_train_smote)
In [274]:
class_report(y_train_new,svc_smote.predict(x_train_new))
In [275]:
class_report(y_test_new,svc_smote.predict(x_test_new))
In [276]:
confusionmatrix(y_train_new,svc_smote.predict(x_train_new))
In [277]:
confusionmatrix(y_test_new,svc_smote.predict(x_test_new))
In [278]:
roc_plot(svc_smote,x_train_new,y_train_new)
In [279]:
roc_plot(svc_smote,x_test_new,y_test_new)
In [280]:
svc_smote_nl=SVC(kernel='poly',degree=3,probability=True)
In [281]:
svc_smote_nl=svc_smote_nl.fit(x_train_smote,y_train_smote)
In [282]:
class_report(y_train_new,svc_smote_nl.predict(x_train_new))
In [283]:
class_report(y_test_new,svc_smote_nl.predict(x_test_new))
In [284]:
confusionmatrix(y_train_new,svc_smote_nl.predict(x_train_new))
In [285]:
confusionmatrix(y_test_new,svc_smote_nl.predict(x_test_new))
In [286]:
roc_plot(svc_smote_nl,x_train_new,y_train_new)
In [287]:
roc_plot(svc_smote_nl,x_test_new,y_test_new)
In [288]:
#WITHOUT USING SMOTE
In [289]:
svc_normal=SVC(probability=True)
svc_normal=svc_normal.fit(x_train_new,y_train_new)
In [290]:
class_report(y_test_new,svc_normal.predict(x_test_new))
confusionmatrix(y_test_new,svc_normal.predict(x_test_new))
In [291]:
class_report(y_train_new,svc_normal.predict(x_train_new))
confusionmatrix(y_train_new,svc_normal.predict(x_train_new))
In [292]:
roc_plot(svc_normal,x_test_new,y_test_new)
In [293]:
roc_plot(svc_normal,x_train_new,y_train_new)
Naive Bayes¶
In [294]:
from sklearn.naive_bayes import GaussianNB
In [295]:
gnb=GaussianNB()
In [296]:
gnb=gnb.fit(x_train_smote,y_train_smote)
In [297]:
class_report(y_train_new,gnb.predict(x_train_new))
In [298]:
confusionmatrix(y_train_new,gnb.predict(x_train_new))
In [299]:
class_report(y_test_new,gnb.predict(x_test_new))
In [300]:
confusionmatrix(y_test_new,gnb.predict(x_test_new))
In [301]:
roc_plot(gnb,x_train_new,y_train_new)
In [302]:
roc_plot(gnb,x_test_new,y_test_new)
In [303]:
gnb_normal=GaussianNB()
gnb_normal=gnb_normal.fit(x_train_new,y_train_new)
In [304]:
class_report(y_test_new,gnb_normal.predict(x_test_new))
confusionmatrix(y_test_new,gnb_normal.predict(x_test_new))
In [305]:
class_report(y_train_new,gnb_normal.predict(x_train_new))
confusionmatrix(y_train_new,gnb_normal.predict(x_train_new))
In [306]:
roc_plot(gnb_normal,x_test_new,y_test_new)
roc_plot(gnb_normal,x_train_new,y_train_new)
Random Forest¶
In [307]:
from sklearn.ensemble import RandomForestClassifier
In [308]:
rfc=RandomForestClassifier(n_estimators=1000,
criterion='gini',
max_depth=None,
min_samples_split=100,
min_samples_leaf=50,
min_weight_fraction_leaf=0.0,
max_features='auto',
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
bootstrap=True,
oob_score=False,
n_jobs=None,
random_state=0,
verbose=0,
warm_start=False,
class_weight=None,
ccp_alpha=0.0,
max_samples=None,)
In [309]:
rfc=rfc.fit(x_train_smote,y_train_smote)
In [ ]:
class_report(y_train_new,rfc.predict(x_train_new))
In [ ]:
class_report(y_test_new,rfc.predict(x_test_new))
In [ ]:
from sklearn.model_selection import GridSearchCV
In [ ]:
param_grid = {
'min_samples_split' : [50,70,100],
'min_samples_leaf':[40,50,60],
'max_depth':[10,15,20],
'random_state' : [0]
}
In [ ]:
RF_model=RandomForestClassifier()
In [ ]:
grid_search=GridSearchCV(estimator=RF_model,param_grid=param_grid,cv=10)
In [ ]:
grid_search.fit(x_train_smote,y_train_smote)
In [1587]:
grid_search.best_estimator_
Out[1587]:
In [310]:
RF_model=RandomForestClassifier(max_depth=15, min_samples_leaf=40, min_samples_split=50,
random_state=0)
In [311]:
RF_model.fit(x_train_smote, y_train_smote)
Out[311]:
In [312]:
## Performance Matrix on train data set
y_train_predict = RF_model.predict(x_train_new)
model_score =RF_model.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [313]:
print('Scores and Curve for Training data is ')
roc_plot(RF_model,x_train_new,y_train_new)
In [314]:
## Performance Matrix on test data set
y_test_predict = RF_model.predict(x_test_new)
model_score = RF_model.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test_new, y_test_predict)
class_report(y_test_new, y_test_predict)
In [315]:
print('Scores and Curve for Testing data is ')
roc_plot(RF_model,x_test_new,y_test_new)
In [316]:
pd.DataFrame(100*(RF_model.feature_importances_),x_train_new.columns)
Out[316]:
In [317]:
plt.figure(figsize=(10,8))
sns.barplot(y=x_train_new.columns,x=100*(RF_model.feature_importances_))
Out[317]:
In [318]:
RF_model_normal=RandomForestClassifier(max_depth=15, min_samples_leaf=40, min_samples_split=50,
random_state=0)
RF_model_normal=RF_model_normal.fit(x_train_new,y_train_new)
In [319]:
class_report(y_train_new,RF_model_normal.predict(x_train_new))
confusionmatrix(y_train_new,RF_model_normal.predict(x_train_new))
In [320]:
class_report(y_test_new,RF_model_normal.predict(x_test_new))
confusionmatrix(y_test_new,RF_model_normal.predict(x_test_new))
In [321]:
roc_plot(RF_model_normal,x_test_new,y_test_new)
roc_plot(RF_model_normal,x_train_new,y_train_new)
Ada-boosting - Using Cross Validation¶
In [322]:
from sklearn.ensemble import AdaBoostClassifier
In [1596]:
param_grid = {
'n_estimators' : [100,500,1000],
'learning_rate' : [0.1,0.01,0.001],
'algorithm' : ['SAMME', 'SAMME.R']
}
In [1597]:
ADB_model=AdaBoostClassifier()
In [1598]:
grid_search=GridSearchCV(estimator=ADB_model,param_grid=param_grid)
In [1599]:
grid_search.fit(x_train_smote,y_train_smote)
Out[1599]:
In [1644]:
grid_search.best_estimator_
Out[1644]:
In [323]:
ADB_model=AdaBoostClassifier(learning_rate=0.1, n_estimators=1000)
In [324]:
ADB_model
Out[324]:
In [325]:
ADB_model.fit(x_train_new,y_train_new)
Out[325]:
In [326]:
## Performance Matrix on train data set
y_train_predict = ADB_model.predict(x_train_new)
model_score = ADB_model.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [327]:
## Performance Matrix on test data set
y_train_predict = ADB_model.predict(x_test_new)
model_score = ADB_model.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test_new, ADB_model.predict(x_test_new))
class_report(y_test_new, ADB_model.predict(x_test_new))
In [328]:
roc_plot(ADB_model,x_train_new,y_train_new)
In [329]:
roc_plot(ADB_model,x_test_new,y_test_new)
In [330]:
pd.DataFrame(100*(ADB_model.feature_importances_),x_train_new.columns)
Out[330]:
In [331]:
ADB_model_normal=AdaBoostClassifier(learning_rate=0.1, n_estimators=1000)
ADB_model_normal=ADB_model_normal.fit(x_train_new,y_train_new)
In [332]:
class_report(y_train_new,RF_model_normal.predict(x_train_new))
confusionmatrix(y_train_new,RF_model_normal.predict(x_train_new))
In [333]:
class_report(y_test_new,ADB_model_normal.predict(x_test_new))
confusionmatrix(y_test_new,ADB_model_normal.predict(x_test_new))
In [336]:
roc_plot(ADB_model_normal,x_train_new,y_train_new)
roc_plot(ADB_model_normal,x_test_new,y_test_new)
In [337]:
y_train_new.value_counts(normalize=True)
Out[337]:
KNN¶
In [338]:
from sklearn.neighbors import KNeighborsClassifier
In [339]:
KNN=KNeighborsClassifier()
KNN=KNN.fit(x_train_smote,y_train_smote)
In [340]:
## Performance Matrix on train data set
y_train_predict = KNN.predict(x_train_new)
model_score = KNN.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [345]:
## Performance Matrix on test data set
y_train_predict = KNN.predict(x_test_new)
model_score = KNN.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test_new, KNN.predict(x_test_new))
class_report(y_test_new, KNN.predict(x_test_new))
In [346]:
roc_plot(KNN,x_train_new,y_train_new)
In [347]:
roc_plot(KNN,x_test_new,y_test_new)
In [348]:
KNN_normal=KNeighborsClassifier()
KNN_normal=KNN_normal.fit(x_train_new,y_train_new)
In [349]:
## Performance Matrix on train data set
y_train_predict = KNN_normal.predict(x_train_new)
model_score = KNN_normal.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [350]:
## Performance Matrix on test data set
y_train_predict = KNN_normal.predict(x_test_new)
model_score = KNN_normal.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test, KNN_normal.predict(x_test_new))
class_report(y_test, KNN_normal.predict(x_test_new))
In [ ]:
roc_plot(KNN_normal,x_train_new,y_train_new)
In [ ]:
roc_plot(KNN_normal,x_test_new,y_test_new)
XGBoost¶
In [ ]:
import xgboost as xgb
In [1620]:
param_test1 = {
'max_depth':range(3,10,2),
'min_child_weight':range(1,6,2)
}
In [1621]:
gsearch1 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=5,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch1.fit(x_train_smote, y_train_smote)
Out[1621]:
In [1622]:
gsearch1.best_params_,gsearch1.best_score_
Out[1622]:
In [529]:
for i in range(8,12,1):
print(i)
In [1623]:
param_test2 = {
'max_depth':range(8,12,1),
'min_child_weight':range(1,4,1)
}
In [1624]:
gsearch2 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=9,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch2.fit(x_train_smote, y_train_smote)
Out[1624]:
In [1625]:
gsearch2.best_params_,gsearch2.best_score_
Out[1625]:
In [533]:
param_test3 = {
'gamma':[i/10.0 for i in range(0,5)]
}
In [534]:
gsearch3 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=11,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test3, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch3.fit(x_train_smote, y_train_smote)
Out[534]:
In [535]:
gsearch3.best_params_,gsearch3.best_score_
Out[535]:
In [1310]:
xgb2 = xgb.XGBClassifier(
learning_rate =0.1,
n_estimators=1000,
max_depth=11,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
In [1311]:
xgb2.fit(x_train_smote, y_train_smote)
Out[1311]:
In [1312]:
class_report(y_train_new,xgb2.predict(x_train_new))
In [1313]:
class_report(y_test_new,xgb2.predict(x_test_new))
In [542]:
param_test4 = {
'subsample':[i/10.0 for i in range(6,10)],
'colsample_bytree':[i/10.0 for i in range(6,10)]
}
In [543]:
gsearch4 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test4, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch4.fit(x_train_smote, y_train_smote)
Out[543]:
In [544]:
gsearch4.best_params_,gsearch4.best_score_
Out[544]:
In [545]:
param_test5 = {
'subsample':[i/100 for i in range(75,90,5)],
'colsample_bytree':[i/10.0 for i in range(4,7)]
}
In [546]:
gsearch5 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test5, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch5.fit(x_train_smote, y_train_smote)
Out[546]:
In [547]:
gsearch5.best_params_,gsearch5.best_score_
Out[547]:
In [552]:
param_test6 = {
'subsample':[i/100 for i in range(80,90,2)],
'colsample_bytree':[i/100 for i in range(58,65,2)]
}
In [553]:
gsearch6 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test6, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch6.fit(x_train_smote, y_train_smote)
Out[553]:
In [555]:
gsearch6.best_params_,gsearch6.best_score_
Out[555]:
In [556]:
param_test7 = {
'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100]
}
In [557]:
gsearch7= GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test7, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch7.fit(x_train_smote, y_train_smote)
Out[557]:
In [558]:
gsearch7.best_params_,gsearch7.best_score_
Out[558]:
In [562]:
param_test8 = {
'reg_alpha':[0.001,0.002,0.003,0.004 ]
}
In [563]:
gsearch8= GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test8, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch8.fit(x_train_smote, y_train_smote)
Out[563]:
In [564]:
gsearch8.best_params_,gsearch8.best_score_
Out[564]:
In [1610]:
xgb3=xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=0.001,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
In [1611]:
xgb3=xgb3.fit(x_train_smote,y_train_smote)
In [1316]:
class_report(y_test_new,xgb3.predict(x_test_new))
In [1317]:
class_report(y_train_new,xgb3.predict(x_train_new))
In [573]:
param_test9 = {
'learning_rate':[0.01,0.02,0.1]
}
In [574]:
gsearch9= GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,
min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test9, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch9.fit(x_train_smote, y_train_smote)
Out[574]:
In [575]:
gsearch9.best_params_,gsearch9.best_score_
Out[575]:
In [1318]:
xgb4=xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,booster='gbtree',
min_child_weight=44, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
xgb4=xgb4.fit(x_train_smote, y_train_smote)
In [1319]:
class_report(y_train_new,xgb4.predict(x_train_new))
In [1320]:
class_report(y_test_new,xgb4.predict(x_test_new))
In [1323]:
confusionmatrix(y_train_new,xgb4.predict(x_train_new))
In [1321]:
confusionmatrix(y_test_new,xgb4.predict(x_test_new))
In [1324]:
print('Scores and Curve for Testing data is ')
roc_plot(xgb4,x_test_new,y_test_new)
In [1325]:
print('Scores and Curve for Training data is ')
roc_plot(xgb4,x_train_new,y_train_new)
In [1326]:
feature_importance=xgb4.get_booster().get_score(importance_type="gain")
feature_col = list(feature_importance.keys())
feature_value =list(feature_importance.values())
In [ ]:
[feature_value]
In [1327]:
plt.figure(figsize=(10,5))
sns.barplot(x=100*(xgb4.feature_importances_),y=x_train_smote.columns)
Out[1327]:
In [673]:
xgb4.get_booster().get_score(importance_type='weight')
Out[673]:
In [1657]:
xgb4_normal=xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,booster='gbtree',
min_child_weight=44, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
xgb4_normal=xgb4_normal.fit(x_train_new, y_train_new)
In [1658]:
## Performance Matrix on train data set
y_train_predict = xgb4_normal.predict(x_train_new)
model_score = xgb4_normal.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [1659]:
## Performance Matrix on test data set
y_test_predict = xgb4_normal.predict(x_test_new)
model_score = xgb4_normal.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test, xgb4_normal.predict(x_test_new))
class_report(y_test, xgb4_normal.predict(x_test_new))
In [1683]:
roc_plot(xgb4_normal,x_train_new,y_train_new)
In [1684]:
roc_plot(xgb4_normal,x_test_new,y_test_new)
Linear Discriminant Analysis¶
In [1612]:
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
In [1614]:
lda=LinearDiscriminantAnalysis()
In [1615]:
lda=lda.fit(x_train_smote, y_train_smote)
In [1617]:
## Performance Matrix on train data set
y_train_predict = lda.predict(x_train_new)
model_score = lda.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [1616]:
## Performance Matrix on test data set
y_train_predict = lda.predict(x_test_new)
model_score = lda.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test, lda.predict(x_test_new))
class_report(y_test, lda.predict(x_test_new))
In [1618]:
roc_plot(lda,x_test_new, y_test_new)
In [1619]:
roc_plot(lda,x_train_new, y_train_new)
In [1660]:
lda_normal=LinearDiscriminantAnalysis()
lda_normal=lda_normal.fit(x_train_new, y_train_new)
In [1661]:
## Performance Matrix on train data set
y_train_predict = lda_normal.predict(x_train_new)
model_score = lda_normal.score(x_train_new, y_train_new)
print(model_score)
confusionmatrix(y_train_new, y_train_predict)
class_report(y_train_new, y_train_predict)
In [1662]:
## Performance Matrix on test data set
y_test_predict = lda_normal.predict(x_test_new)
model_score = lda_normal.score(x_test_new, y_test_new)
print(model_score)
confusionmatrix(y_test, lda_normal.predict(x_test_new))
class_report(y_test, lda_normal.predict(x_test_new))
In [1685]:
roc_plot(lda_normal,x_train_new, y_train_new)
roc_plot(lda_normal,x_test_new, y_test_new)
In [ ]:
In [ ]:
BELOW CODES NOT TO BE CONSIDERED FOR EVALUATION¶
2nd Iteration¶
In [342]:
df_2.head()
Out[342]:
In [343]:
df4=df_2.copy()
In [1905]:
from sklearn.model_selection import train_test_split
x=df4.drop(['FORECLOSURE'],axis=1)
y=df4.pop('FORECLOSURE')
x_new=x.copy()
y_new=y.copy()
x_train_new,x_test_new,y_train_new,y_test_new=train_test_split(x_new,y_new,test_size=0.3,random_state=0)
#/**x_new_copy=pd.DataFrame()
#x_new_copy['LAP']=x_train_new['LAP']
#x_new_copy['STHL']=x_train_new['STHL']
#x_new_copy['STLAP']=x_train_new['STLAP']
#x_new_copy.head(2)
#x_new_copy.reset_index(drop=True,inplace=True)''''''**/
In [1906]:
### Z score Transformation
from sklearn.preprocessing import StandardScaler
x_train_new.drop(['const'],axis=1,inplace=True)
x_test_new.drop(['const'],axis=1,inplace=True)
x_train_new.shape
Out[1906]:
In [1907]:
list_col=['COMPLETED_TENURE', 'DIFF_CURRENT_INTEREST_RATE_MAX_MIN', \
'DIFF_ORIGINAL_CURRENT_INTEREST_RATE', 'EMI_DUEAMT', \
'LATEST_TRANSACTION_MONTH', 'Mean_Interest_Rate', 'PAID_AMOUNT', 'LAP', \
'STLAP']
for col in x_train_new.columns:
if col not in list_col:
x_train_new.drop([col],axis=1,inplace=True)
x_test_new.drop([col],axis=1,inplace=True)
In [1908]:
print(x_train_new.shape)
print(x_test_new.shape)
In [1909]:
x_train_new.head(2)
Out[1909]:
In [1910]:
from sklearn.preprocessing import StandardScaler
ss = StandardScaler()
x_train_new=pd.DataFrame(ss.fit_transform(x_train_new),columns = x_train_new.columns)
In [1911]:
x_train_new.head()
Out[1911]:
In [1717]:
x_test_new.head()
Out[1717]:
In [1718]:
x_train_smote.head()
Out[1718]:
In [1728]:
RF_model.feature_importances_.T
Out[1728]:
In [1781]:
feature_important=pd.DataFrame(columns=x_train_smote.columns)
In [1784]:
i=0
for col in feature_important.columns:
feature_important.loc[0,col]=100* RF_model.feature_importances_[i]
i = i+1
In [1777]:
for col in feature_important.columns:
print(col)
In [1765]:
#feature_important.loc[0,'BALANCE_TENURE']=RF_model.feature_importances_[0]
In [1799]:
feature_important.columns
Out[1799]:
In [1803]:
feature_important.shape
Out[1803]:
In [1791]:
[feature_important.drop([col],axis=1,inplace=True) for col in feature_important.columns if feature_important.loc[0,col] < 3]
Out[1791]:
In [1809]:
x_train_new_copy=x_train_new.copy()
x_test_new_copy=x_test_new.copy()
x_train_smote_copy=x_train_smote.copy()
In [1810]:
list_col=['COMPLETED_TENURE', 'DIFF_CURRENT_INTEREST_RATE_MAX_MIN', \
'DIFF_ORIGINAL_CURRENT_INTEREST_RATE', 'EMI_DUEAMT', \
'LATEST_TRANSACTION_MONTH', 'Mean_Interest_Rate', 'PAID_AMOUNT', 'LAP', \
'STLAP']
for col in x_train_new_copy.columns:
if col not in list_col:
x_train_new_copy.drop([col],axis=1,inplace=True)
x_test_new_copy.drop([col],axis=1,inplace=True)
x_train_smote_copy.drop([col],axis=1,inplace=True)
In [1920]:
df_2[['COMPLETED_TENURE', 'DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE', 'EMI_DUEAMT', \
'LATEST_TRANSACTION_MONTH', 'Mean_Interest_Rate', 'PAID_AMOUNT', 'LAP','STLAP','FORECLOSURE']].head()
Out[1920]:
In [1582]:
from sklearn.model_selection import GridSearchCV
In [1818]:
param_grid = {
'min_samples_split' : [40,50,55],
'min_samples_leaf':[30,40,45],
'max_depth':[10,15,20],
'random_state' : [0]
}
In [1819]:
rf_deploy=RandomForestClassifier()
In [1820]:
grid_search_deploy=GridSearchCV(estimator=rf_deploy,param_grid=param_grid,cv=10)
In [1821]:
grid_search_deploy.fit(x_train_smote_copy,y_train_smote)
Out[1821]:
In [1587]:
grid_search_deploy.best_estimator_
Out[1587]:
In [ ]:
ss.inverse_transform
In [1822]:
rf_deploy=RandomForestClassifier(max_depth=15, min_samples_leaf=40, min_samples_split=50,
random_state=0)
In [1823]:
rf_deploy=rf_deploy.fit(x_train_smote_copy,y_train_smote)
In [1824]:
class_report(y_train_new,rf_deploy.predict(x_train_new_copy))
In [1825]:
class_report(y_test_new,rf_deploy.predict(x_test_new_copy))
In [1827]:
confusionmatrix(y_train_new,rf_deploy.predict(x_train_new_copy))
In [1826]:
confusionmatrix(y_test_new,rf_deploy.predict(x_test_new_copy))
In [1842]:
x_test_new_copy.head(1)
Out[1842]:
In [1852]:
df_2.head(2)
Out[1852]:
In [1853]:
x_train_new_copy.head(3)
Out[1853]:
In [1865]:
ss.mean_.shape
Out[1865]:
In [1866]:
a=x_train_new.copy()
In [1868]:
a.drop(['LAP','STHL','STLAP'],axis=1,inplace=True)
In [1880]:
scaling_features=pd.DataFrame(data=ss.mean_.reshape(1,24) , columns=a.columns)
In [1882]:
scaling_features.loc[1]=ss.var_
In [1883]:
scaling_features
Out[1883]:
In [1884]:
scaling_features.iloc[1]=np.sqrt(scaling_features.iloc[1])
In [1885]:
scaling_features
Out[1885]:
In [1886]:
list_col=['COMPLETED_TENURE', 'DIFF_CURRENT_INTEREST_RATE_MAX_MIN', \
'DIFF_ORIGINAL_CURRENT_INTEREST_RATE', 'EMI_DUEAMT', \
'LATEST_TRANSACTION_MONTH', 'Mean_Interest_Rate', 'PAID_AMOUNT', 'LAP', \
'STLAP']
[scaling_features.drop([col],axis=1,inplace=True) for col in scaling_features.columns if col not in list_col ]
Out[1886]:
In [1888]:
scaling_features.head()
Out[1888]:
Deployment¶
In [1828]:
import pickle
In [1830]:
ss.mean_
Out[1830]:
In [1835]:
rf_file=open('rf.pkl','wb')
pickle.dump(rf_deploy,rf_file)
loaded_model=pickle.load(open('rf.pkl','rb'))
print(loaded_model.score(x_train_new_copy,y_train_new))
In [1912]:
pickle.dump(ss,open('scaler.pkl','wb'))
In [1840]:
rf_deploy.feature_importances_.reshape(9,1)
Out[1840]:
In [1839]:
rf_deploy.feature_importances_
Out[1839]:
In [1846]:
type([[0.626971,0.622898,0.6142,0.002305,2.02789,0.869787,0.03947,1,0]])
Out[1846]:
In [1811]:
x_train_new_copy.shape
Out[1811]:
In [1813]:
x_train_smote_copy.head()
Out[1813]:
In [1748]:
RF_model.feature_importances_[0]
Out[1748]:
In [1729]:
pd.DataFrame(100*(RF_model.feature_importances_.T))#.T,columns=x_train_smote.columns
Out[1729]:
In [1724]:
x_train_smote.columns
Out[1724]:
In [945]:
x_new=x.copy()
y_new=y.copy()
In [946]:
x_new['LAP']=df_2['LAP']
x_new['STHL']=df_2['STHL']
x_new['STLAP']=df_2['STLAP']
In [947]:
#x_new.drop(['LATEST_TRANSACTION_MONTH'],axis=1,inplace=True)
In [948]:
x_train_new,x_test_new,y_train_new,y_test_new=train_test_split(x_new,y_new,test_size=0.3,random_state=0)
In [949]:
#Validation Set
#x_test_1,x_test_final,y_test_1,y_test_final=train_test_split(x_test_new,y_test_new,test_size=0.3,random_state=0)
In [950]:
sm_new = SMOTE(random_state=0)
x_train_new,y_train_new=sm_new.fit_resample(x_train_new,y_train_new)
In [1059]:
x_train_new.head()
Out[1059]:
In [1060]:
y_train_new.head()
Out[1060]:
In [1061]:
xgb4_new=xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,booster='gbtree',
min_child_weight=44, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
xgb4_new=xgb4_new.fit(x_train_new, y_train_new)#44
In [1062]:
x_train_new.head()
Out[1062]:
In [1069]:
class_report(y_train_new,xgb4_new.predict(x_train_new))
In [1068]:
x_train_new.shape
Out[1068]:
In [1067]:
x_test_new.shape
Out[1067]:
In [1076]:
x_test_new=x_test_new.apply(zscore)
In [1077]:
x_test_new.head()
Out[1077]:
In [1078]:
class_report(y_test_new,xgb4_new.predict(x_test_new)) #88 94 91 98 without latest_Transaction
In [955]:
plt.figure(figsize=(10,5))
sns.barplot(x=100*(xgb4_new.feature_importances_),y=x_test_new.columns)
Out[955]:
In [973]:
x_new_smote=x_train_smote[['LATEST_TRANSACTION_MONTH','LAP','STHL','STLAP','EMI_DUEAMT','CURRENT_INTEREST_RATE_CHANGES'\
,'DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'Mean_Interest_Rate','DPD','DUEDAY','ORIGNAL_TENOR','PAID_AMOUNT']] #'LATEST_TRANSACTION_MONTH',
In [ ]:
x_train_new_1=x_train_new[['LATEST_TRANSACTION_MONTH','LAP','STHL','STLAP','EMI_DUEAMT','CURRENT_INTEREST_RATE_CHANGES'\
,'DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'Mean_Interest_Rate','DPD','DUEDAY','ORIGNAL_TENOR','PAID_AMOUNT']]
In [974]:
x_test_new_1=x_test_new[['LATEST_TRANSACTION_MONTH','LAP','STHL','STLAP','EMI_DUEAMT','CURRENT_INTEREST_RATE_CHANGES'\
,'DIFF_CURRENT_INTEREST_RATE_MAX_MIN','DIFF_ORIGINAL_CURRENT_INTEREST_RATE',\
'Mean_Interest_Rate','DPD','DUEDAY','ORIGNAL_TENOR','PAID_AMOUNT']]#'LATEST_TRANSACTION_MONTH',
In [975]:
x_new_smote.head()
Out[975]:
In [976]:
xgb4_new=xgb.XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=11,reg_alpha=1e-05,booster='gbtree',
min_child_weight=44, gamma=0, subsample=0.8, colsample_bytree=0.6,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
xgb4_new=xgb4_new.fit(x_new_smote, y_train_new)
In [977]:
class_report(y_train_new,xgb4_new.predict(x_new_smote))
In [978]:
class_report(y_test_new,xgb4_new.predict(x_test_new_smote)) #83 95 89
In [979]:
confusionmatrix(y_train_new,xgb4_new.predict(x_new_smote))
In [980]:
confusionmatrix(y_test_new,xgb4_new.predict(x_test_new_smote))
In [981]:
plt.figure(figsize=(10,5))
sns.barplot(x=100*(xgb4_new.feature_importances_),y=x_test_new_smote.columns)
Out[981]:
In [982]:
print('Scores and Curve for Training data is ')
roc_plot(xgb4_new,x_new_smote,y_train_new)
In [983]:
print('Scores and Curve for Training data is ')
roc_plot(xgb4_new,x_test_new_smote,y_test_new)
In [1037]:
x_test.head(2)
Out[1037]:
In [ ]: